#! /usr/bin/env python
#author JP Braun Mandriva
# -*- coding: utf-8 -*-

# To do :
#   Short term
#   Next step :
#   - create a trivial db with django with the data of the rpm
#     - vendor, distribution, rpm, version, files, provides, dependencies
#   - update version of the creation : if something exists, do not do it,
#     if it is new, do it.
#   Longer term :
#   - Redhat, Suse
#   - dpkg (so trivial to do !!!)

from Cheetah.Template import Template
from Languages import *
from popen2 import *
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer, get_lexer_for_filename, get_lexer_for_mimetype
from Util import *
from xml.sax.saxutils import escape, quoteattr
import Queue
import cgi
import getopt
import locale
import os
import re
import rpm
import shutil
import sys
import threading
import time
import traceback
#from threadpool import * 

locale.setlocale(locale.LC_ALL, 'C')


class File:
    def __init__(self, package, file, output_file, cat, log):
       
        self.package = package
        self.file = file
        self.output_file = output_file
        self.path = os.path.split(self.file)[0]
        self.output_path = os.path.split(self.output_file)[0]
        self.name = os.path.split(self.file)[1]
        self.output_name = os.path.split(self.output_file)[1]
        self.log_path = log
        self.ext = os.path.splitext(self.output_name)[1]
        self.download = '/'
        if self.path.startswith('/'):
            path = self.path[1:]
        else:
            path = self.path
        self.url = 'url'
        #os.path.join(package.url_content_path,path,self.output_name + '.doc4.html')
        self.html = self.output_file + '.doc4.html'
        self.preview = ''
        self.cat = cat
        if os.path.isfile(self.output_file):
            self.size = os.path.getsize(os.path.join(self.output_path, self.name))
            self.rsize = os.path.getsize(self.output_file)
        else:
            self.size = 0
            self.rsize = 0

        #        file = self.output_file
        #        print file, "=>",
        #        if os.path.exists(file):
        #            print "EXISTS",
        #        if os.path.isdir(file):
        #            print "ISDIR",
        #        if os.path.isfile(file):
        #            print "ISFILE",
        #        if os.path.islink(file):
        #            print "ISLINK",
        #        if os.path.ismount(file):
        #            print "ISMOUNT",
        #        print
            
        # get type of file
        if os.path.isdir(self.output_file):
            # set permissions
            os.chmod(self.output_file, 0755)
            self.type = 'dir'
        elif os.path.islink(self.output_file):
            self.type = 'link'
        elif os.path.isfile(self.output_file):

            # set permissions
            os.chmod(self.output_file, 0644)

            # man file #
            if self.ext in ('.1', '.2', '.3', '.4', '.5', '.6', '.7', '.8', '.9') and 'man' in self.output_file:
                self.type = 'man'
            # info file #
            elif self.ext == '.info':
                self.type = 'info'
            # html file #
            elif self.ext == '.html' or self.ext == '.htm':
                self.type = 'html'
            # translation file #
            elif self.ext == '.mo' or self.ext == '.po':
                self.type = 'translation'
                self.cat = 'translation'
                self.package.nb_trans_files += 1
            # docbook #
            elif self.ext == '.docbook':
                self.type = 'docbook'
            # code #
            elif self.ext in ('.ada', '.awk', '.c', '.css', '.h', '.lua', '.cpp', '.c++', '.h++', '.cc', '.cxx', '.hxx', '.asm', '.groff', '.php', '.java', '.js', '.lisp', '.el', '.cl', '.erl', '.hrl', '.hs', '.m4', '.pas', '.patch', '.diff', '.pm', '.pod', '.pl', '.pov', '.py', '.rb', '.sh', '.tcsh', '.csh', '.sql', '.ml', '.mli', '.mll', '.mly', '.scm', '.bat', '.cmd', '.plot', '.plt', '.xsl', '.xml', '.tex', '.aux', '.toc', '.as', '.rss', '.xslt', '.xsd', '.wsdl', '.pyw', '.sc', '.pytb', '.tcl') or self.name in ('Makefile'):
                self.type = 'code'
            # image #
            elif self.ext in ('.svg', '.png', '.jpg', '.jpeg', '.bmp', '.gif'):
                self.type = 'image'
            # audio #
            elif self.ext in ('.ogg', '.wav', '.mp3'):
                self.type = 'audio'
            # text file #
            elif os.path.isfile(self.output_file) and os.popen("file -i '" + self.output_file + "' | grep ':\ application'").read() == '':
                self.type = 'text'
            else:
                self.type = 'binary'
        else:
            self.type = 'unknow'
            try:
                os.unlink(self.output_file)
            except OSError:
                pass
            log_package("\nNot a regular file : " + self.file, self.log_path)

    def extract(self):
        return
            
    def extract_audio(self):
        mp3_file = self.output_file.rsplit('.', 1)[0] + ".mp3"
        # convert file in mp3
        try:
            if self.ext == '.ogg':
                pid = Popen3("ogg123 -q -d wav -f - '" + self.output_file + "' | lame -f --quiet - > '" + mp3_file + "'")
                sts = pid.wait()
            else:
                pid = Popen3("lame -f --quiet '" + self.output_file + "' '" + mp3_file + "'")
                sts = pid.wait()
        except Exception, e:
            log_package("audio convert failed in extract_audio - " + str(e), self.log_path)
        
        content = '<h2>' + self.file + '</h2>'
        content += '<object type="application/x-shockwave-flash" data="/inc/dewplayer.swf?mp3=' + os.path.basename(mp3_file) + '" width="200" height="20"><param name="movie" value="/inc/dewplayer.swf?mp3' + os.path.basename(mp3_file) + '" /></object>'; 
        write_string(content, self.html)


    def __str__(self):
        print 'file : ' + self.file
        print 'output file : ' + self.output_file
        print 'file path : ' + self.path
        print 'output file path : ' + self.output_path
        print 'file name : ' + self.name
        print 'output file name : ' + self.output_name
        print 'category : ' + self.cat
        print 'html file : ' + self.html
        print 'file ext : ' + self.ext
        print 'download : ' + self.download
        print 'url : ' + self.url
        print 'type : ' + self.type
        print 'preview : ' + self.preview
        print 'metadatas : '
        print self.metadatas

        return ''

class Package:
    def __init__(self, source_path, output_path, log):
        self.source_path = os.path.abspath(source_path)
        self.output_path = os.path.abspath(output_path)
        self.log = os.path.abspath(log)
        self.full_name = os.path.basename(self.source_path)
        self.files = []
        self.nb_files = 0
        self.nb_conf_files = 0
        self.nb_doc_files = 0
        self.nb_trans_files = 0
        self.download = ''
        

    def get_source_path(self):
        return self.source_path
    
    def get_output_path(self):
        return self.output_path

    def get_log(self):
        return self.log

    def analyse_files(self):
        for file in self.ql:
            if not 'contains no files' in file:
                file_info = self.analyse_file(file)
                self.files.append({'path': quoteattr(escape(file)), 'data': file_info})        
            else:
                log_package("This must be a meta-package", self.log_path)
                self.nb_files = 0
                self.nb_doc_files = 0
                self.nb_conf_files = 0

        for archive, files in self.extra_files.items():
            cpt = 0
            for file in files:
                file_info = self.analyse_file(file)
                self.extra_files[archive][cpt] = {'path': quoteattr(escape(file)), 'data': file_info}
                cpt += 1

    def analyse_file(self, file):
        file_decomp = file
        # decompress man, info files
        if file.endswith('.lzma') and ('info' in file or 'man' in file):
            output_file = os.path.join(self.output_path, file[1:])
            # test if archive is correct
            pid = Popen3("lzma --force -t '" + output_file + "'")
            sts = pid.wait()
            if sts == 0:
                # decompress
                pid = Popen3("lzma -dk --force '" + output_file + "'")
                sts = pid.wait()
                file_decomp = os.path.splitext(file)[0]

        if file_decomp.startswith('/'):
            file_decomp = file_decomp[1:]

        output_file = os.path.join(self.output_path, file_decomp)
    
        # test if file is a conf or doc file
        if file in self.qd:
            cat = "documentation"
        elif file in self.qc:
            cat = "configuration"
        else:
            cat = ""
        # create File object
        file_info = File(self, file, output_file, cat, self.log)

        return file_info
    
    def process_files(self):
        for file in self.files:
            #log_package('\nfile:'+file['path'], log_path)
            file['data'].extract()        
        for archive, files in self.extra_files.items():
            for file in files:
                #log_package('\nfile:'+file['path'], log_path)
                file['data'].extract()        

class Rpm(Package):
    def __init__(self, source_path, output_path, log):
        Package.__init__(self, source_path, output_path, log)
        if source_path.endswith('src.rpm'):
            self.type = 'srpm'
        else:
            self.type = 'rpm'

        ts = rpm.ts()
        ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
        fdno = os.open(self.source_path, os.O_RDONLY)
        # read RPM header
        hdr = ts.hdrFromFdno(fdno)
        os.close(fdno)

        # get some tags
        self.taglist = {}
        self.taglist['NAME'] = hdr[rpm.RPMTAG_NAME]
        self.taglist['SOURCERPM'] = hdr[rpm.RPMTAG_SOURCERPM]
        self.taglist['SUMMARY'] = hdr[rpm.RPMTAG_SUMMARY]
        self.taglist['URL'] = hdr[rpm.RPMTAG_URL]
        self.taglist['VERSION'] = hdr[rpm.RPMTAG_VERSION]
        self.taglist['RELEASE'] = hdr[rpm.RPMTAG_RELEASE]
        self.taglist['RPMVERSION'] = hdr[rpm.RPMTAG_RPMVERSION]
        self.taglist['OS'] = hdr[rpm.RPMTAG_OS]
        self.taglist['DISTRIBUTION'] = hdr[rpm.RPMTAG_DISTRIBUTION]
        self.taglist['LICENSE'] = hdr[rpm.RPMTAG_LICENSE]
        self.taglist['ARCH'] = hdr[rpm.RPMTAG_ARCH]
        self.taglist['SIZE'] = hdr[rpm.RPMTAG_SIZE]
        self.taglist['ARCHIVESIZE'] = hdr[rpm.RPMTAG_ARCHIVESIZE]
        self.taglist['GROUP'] = hdr[rpm.RPMTAG_GROUP]
        #The tag COPYRIGHT is not available in python-rpm 4.6
        #-> we consider COPYRIGHT=LICENSE for now
        #self.taglist['COPYRIGHT'] = hdr[rpm.RPMTAG_COPYRIGHT]
        self.taglist['COPYRIGHT'] = hdr[rpm.RPMTAG_LICENSE]
        self.taglist['PACKAGER'] = hdr[rpm.RPMTAG_PACKAGER]
        self.taglist['BUILDTIME'] = hdr[rpm.RPMTAG_BUILDTIME]
        self.taglist['BUILDHOST'] = hdr[rpm.RPMTAG_BUILDHOST]
        self.taglist['REQUIRES'] = unique(hdr[rpm.RPMTAG_REQUIRES])
        self.taglist['PROVIDES'] = hdr[rpm.RPMTAG_PROVIDES]
        self.taglist['DESCRIPTION'] = hdr[rpm.RPMTAG_DESCRIPTION]
        self.taglist['PREIN'] = hdr[rpm.RPMTAG_PREIN]
        self.taglist['POSTIN'] = hdr[rpm.RPMTAG_POSTIN]
        self.taglist['PREUN'] = hdr[rpm.RPMTAG_PREUN]
        self.taglist['POSTUN'] = hdr[rpm.RPMTAG_POSTUN]
        self.taglist['CONFLICTS'] = hdr[rpm.RPMTAG_CONFLICTS]
        self.taglist['CONFLICTVERSION'] = hdr[rpm.RPMTAG_CONFLICTVERSION]
        self.taglist['FILENAMES'] = hdr[rpm.RPMTAG_FILENAMES]
        self.taglist['OBSOLETES'] = hdr[rpm.RPMTAG_OBSOLETENAME]
        self.taglist['SUGGESTS'] = hdr[rpm.RPMTAG_SUGGESTSNAME]
        self.taglist['SUGGESTSVERSION'] = hdr[rpm.RPMTAG_SUGGESTSVERSION]


        if hdr[rpm.RPMTAG_CHANGELOGTEXT] and hdr[rpm.RPMTAG_CHANGELOGNAME]:
            self.taglist['CHANGELOG'] = ""
            for clidx in range(0, len(hdr[rpm.RPMTAG_CHANGELOGTEXT])):
                # I've seen yum headers where changelogs with a single entry have the timestamp (but not the other stuff)
                # stored as a string instead of a list. Tres annoying. =:\
                try:
                    # Generates 'TypeError' exception on string
                    len(hdr[rpm.RPMTAG_CHANGELOGTIME])
                    timestamp = time.ctime(hdr[rpm.RPMTAG_CHANGELOGTIME][clidx])
                except TypeError:
                    timestamp = time.ctime(hdr[rpm.RPMTAG_CHANGELOGTIME])
                    
                self.taglist['CHANGELOG'] = self.taglist['CHANGELOG'] + "* %s - %s\n%s\n\n" % (timestamp, hdr[rpm.RPMTAG_CHANGELOGNAME][clidx], hdr[rpm.RPMTAG_CHANGELOGTEXT][clidx])
        else:
            self.taglist['CHANGELOG'] = 'none'

        # 17 = conf
        # 2 = doc
        flags = hdr['fileflags']
        # files list
        self.ql = hdr['filenames']
        # get the conf & doc files from files list
        self.qd = []
        self.qc = []
        for file, flag in zip(self.ql, flags):
            if flag == 17:
                self.qc.append(file)
            if flag == 2:
                self.qd.append(file)
        
        #print "self.qd"+self.qd
        self.nb_conf_files = len(self.qc)
        self.nb_doc_files = len(self.qd)
        self.nb_files = len(self.ql)
        self.extra_files = {}
    
    def extract_files(self, extraction_home):
        p1 = Popen3('cd ' + self.output_path + '; rpm2cpio ' + self.source_path + ' | cpio -id --quiet ')
        sts = p1.wait()

        if sts != 0:
            log_package('\nError decompressing ' + self.source_path, self.log)
            raise DecompressingError()
        else:
            #log_package('\nDecomp of '+os.path.basename(self.source_path)+' done.', self.log_path)
            print ''

        # we decompress files in src.rpm
        if self.type == 'srpm':
            for file in self.ql:
                if file.endswith('.tgz') or file.endswith('.tar.gz'):
                    cmd = 'tar xvfz ' + self.output_path + '/' + file + ' -C ' + self.output_path
                    self.extract_extrafiles(cmd, file)
                elif file.endswith('.tar.bz2'):
                    cmd = 'tar xvfj ' + self.output_path + '/' + file + ' -C ' + self.output_path
                    self.extract_extrafiles(cmd, file)
                elif file.endswith('.tar.lzma'):
                    cmd = 'tar xvf ' + self.output_path + '/' + file + ' -C ' + self.output_path + ' --lzma'
                    self.extract_extrafiles(cmd, file)
#                elif file.endswith('.bz2'):
#                    cmd = 'bunzip2 '+self.output_path+'/'+file+' -C '+self.output_path
#                    self.extract_extrafiles(cmd, file)

    def extract_extrafiles(self, cmd, file):
        f = os.popen(cmd)
        data = f.readlines()
        sts = f.close()
        # decompress ok
        if sts == None:
            self.extra_files[file] = []
            for decomp_file in data:
                self.extra_files[file].append(decomp_file.strip())

    def get_tag(self, tag):
        return self.taglist[tag]
    def get_ql(self):
        return self.ql
    def get_qd(self):
        return self.qd
    def get_qc(self):
        return self.qc

class DecompressingError(Exception):
    pass


class RpmExtractor:
    
    def export_metadata(self, package, repository_id, branch, section, extraction_home):
        content = ''
        # get package infos
        content += self.get_metadata(package, repository_id, branch, section, extraction_home)
        metadata_path = os.path.join(package.output_path, 'metadata.xml')
        if (os.path.exists(metadata_path)):
            os.remove(metadata_path)
        write_string(content, metadata_path, False)

    def get_template(self, file_name, extraction_home):
        path = extraction_home + '/' + file_name
        tmpl = Template(file=path, filter='EncodeUnicode')
        return tmpl

    def extract_package(self, source_path, output_path, metadata_only, repository_id, branch, section, extraction_home, log):
        
        if not os.path.exists(output_path):
            os.makedirs(output_path, 0755)
        
        ttmp = source_path.split('/')
        package_fullname = source_path
        if (len(ttmp) > 0):
            package_fullname = ttmp[len(ttmp) - 1]
        
        begin_time = time.time()
        
        log_package(make_column(package_fullname, 80), log)
        log_package("" + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(begin_time)), log)
      
        # we instantiate the rpm object
        #print "Source: %s Output: %s" % (source_path, output_path)
        package = Rpm(source_path, output_path, log)
    
        if not metadata_only:
            log_package('Extracting files', log)
            package.extract_files(extraction_home)
            
            # detect package files types, make urls
            log_package('Analysing files', log)
            package.analyse_files()
        
            log_package('Processing files', log)
            package.process_files()
    
        self.export_metadata(package, repository_id, branch, section, extraction_home)
        
        #if package.type == "rpm":
        #    export_configuration_files_metadata(package)
        #    export_documentation_files_metadata(package)
        #    export_translation_files_metadata(package)
        
        end_time = time.time()
        total_time = end_time - begin_time
    
      
        log_package('\t' + time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(end_time)), log)
        log_package('\t' + str(total_time), log)
        end = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(end_time))
        
        return 0
    
    ############################
    # Layout
    ############################
    
    def get_metadata(self, package, repository_id, branch, section, extraction_home):
        #we have to get all the important information about the package, using rpm
        #:SL: is the call to the 'clean' method really needed?
        info = self.get_template('package-metadata.xml', extraction_home)
        info.name = clean(escape(package.get_tag('NAME')), package.log)
        info.full_name = clean(package.full_name, package.log)
        info.download = clean(package.download, package.log)
        info.summary = clean(escape(unicode(package.get_tag('SUMMARY'), encoding="utf-8", errors="ignore")), package.log)
        info.url = escape(clean(package.get_tag('URL'), package.log))
        info.version = package.get_tag('VERSION')
        info.release = package.get_tag('RELEASE')
        info.license = clean(escape(package.get_tag('LICENSE')), package.log)
        info.srpm = clean(package.get_tag('SOURCERPM'), package.log)
        info.rpmversion = package.get_tag('RPMVERSION')
        info.os = clean(escape(package.get_tag('OS')), package.log)
        info.distrib = clean(escape(package.get_tag('DISTRIBUTION')), package.log)
        info.branch = clean(branch, package.log)
        info.section = clean(section, package.log)
        info.repository_id = repository_id
        info.arch = clean(package.get_tag('ARCH'), package.log)
        info.size = package.get_tag('SIZE')
        info.archivesize = package.get_tag('ARCHIVESIZE')
        info.group = clean(package.get_tag('GROUP'), package.log)
        info.copyright = escape(clean(package.get_tag('COPYRIGHT'), package.log))
        packager = package.get_tag('PACKAGER')
        #escape(clean(,package.log))
        info.packager_email = ""
        info.packager = ""
        if not_empty(packager): 
            idx = packager.find("@")
            idxlt = packager.find("<")
            idxgt = packager.find(">")
            if idx > 0 and idxlt > 0 and idxgt > 0 and idxlt < idxgt:
                info.packager = escape(clean(packager[0:idxlt].strip(), package.log))
                info.packager_email = escape(clean(packager[idxlt + 1:idxgt], package.log))
            else:
                info.packager = escape(clean(packager.strip(), package.log))

        info.buildtime = package.get_tag('BUILDTIME')
        info.buildhost = clean(package.get_tag('BUILDHOST'), package.log)
        info.prein = clean(escape_list(package.get_tag('PREIN')), package.log)
        info.postin = clean(escape_list(package.get_tag('POSTIN')), package.log)
        info.preun = clean(escape_list(package.get_tag('PREUN')), package.log)
        info.postun = clean(escape_list(package.get_tag('POSTUN')), package.log)
        
        changelog = clean(package.get_tag('CHANGELOG'), package.log).strip()
        info.changelog = escape(unicode(changelog, encoding="utf-8", errors="ignore"))
    
        info.requires = package.get_tag('REQUIRES')
        info.provides = package.get_tag('PROVIDES')
        info.obsoletes = package.get_tag('OBSOLETES')
        conflicts = package.get_tag('CONFLICTS')
        conflict_versions = package.get_tag('CONFLICTVERSION')
        if (len(conflicts)) == 0:
            info.conflicts = []
        else:
            info.conflicts = zip(conflicts, conflict_versions)
        
        suggests = package.get_tag('SUGGESTS')
        suggests_versions = package.get_tag('SUGGESTSVERSION')
        
        if (len(suggests)) == 0:
            info.suggests = []
        else:
            info.suggests = zip(suggests, suggests_versions)
        
        try:
            description = escape(package.get_tag('DESCRIPTION').strip().encode("utf-8"))
        except Exception, e:
            log_package('*** error: ' + str(e), package.log)
            description = 'An error occurred while decoding the description character stream.'  
        #for bind (presence of escape chars in the description)
        #description = description.replace("\\","\\\\")
        info.description = clean(unicode(description, encoding="utf-8", errors="ignore"), package.log)
        
        info.files = []
        info.extra_files = []
        info.tfiles = package.get_tag('FILENAMES')
        
        if package.files:
            info.tfiles = None
            info.type = package.type
            info.files = package.files 
            info.extra_files = package.extra_files
            info.columns = {}
            
        return str(info)


##############################
# Main loop
##############################

def escapechars(s, chars=[ '\"', '\'', '\\' ]):
    """Converts invalid characters to corresponding html"""
    return "".join(['\\%s' % c if c in chars else c for c in s])


def clean(st, log):
    try:
        if (st == None or len(st) == 0):
            return ""
        else:
            return st
    except Exception, e:
        log_package('\n*** Error: ' + str(e) + '\n', log)
    

def escape_list(list):
    if (list == None):
        return ""
    tmp = ''.join(list)
    #tmp = escape(tmp)
    #tmp = escape(tmp.encode("utf-8"))
    return escape(tmp)
    
def get_conflicts(rpm):
    if len(rpm.get_tag('CONFLICTS')) == 0:
        return 'none'
    content = ''
    for pkg, version in zip(rpm.get_tag('CONFLICTS'), rpm.get_tag('CONFLICTVERSION')):
        content += pkg + ' < ' + version
    return content

def log(string):
    #log_and_print(string, 'doc4.log')
    print "%s" % string

def main():

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hce", ["help", "count", "extract"])
    except getopt.GetoptError:
        pass
        
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            print 'Usage :'
            sys.exit(0)
        elif opt in ("-e", "--extract"):
            pacx = RpmExtractor()
            mirrorPath = "/home/doc4/mirror/mandriva/mandriva/cooker/"
            outputDirHome = "/home/doc4/data"
            scriptsHome = "/home/doc4/doc4packages"
            logFilePath = "/home/doc4/logs/Extraction.log"
            #toextract = "junit-3.8.2-5.0.2mdv2009.1.i586.rpm"
            toextract = "junit-3.8.2-5.0.2mdv2009.1.src.rpm"
            outputDir = os.path.join(outputDirHome, str(string_to_integer(toextract)), toextract)
            print "Output directory: %s" % outputDir
            os.makedirs(outputDir)
            extraction_result = pacx.extract_package(os.path.join(mirrorPath, toextract), outputDir, False, 10, "branch", "section", scriptsHome, logFilePath)

if __name__ == "__main__":
    main()
